﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Common.TextMatch
{
	/// <summary>
	/// 
	/// </summary>
	public class TfIdfUtil
	{
		#region fields
		// phrase -> termId
		private readonly Dictionary<string, int> _Terms;
		// docId -> label (usually doc title)
		private readonly Dictionary<int, string> _DocLabels;
		// docId -> (termId -> freq)
		private readonly Dictionary<int, Dictionary<int, int>> _DocTermFreqs;
		// termid -> term tf-idf weights (average)
		private Dictionary<int, double> _TermWeights;
		#endregion

		#region props
		/// <summary>
		/// unique terms contained in all documents
		/// </summary>
		public Dictionary<string,int> AllTerms
		{
			get
			{
				return _Terms;
			}
		}

		/// <summary>
		/// 
		/// </summary>
		public Dictionary<int,string> AllDocs
		{
			get
			{
				return _DocLabels;
			}
		}
		#endregion

		#region lookup 
		/// <summary>
		/// tfidf weight
		/// </summary>
		/// <param name="phrase"></param>
		/// <returns></returns>
		public double GetTermWeight(string phrase)
		{
			if (this._Terms.ContainsKey(phrase.ToLower()))
				return this._TermWeights[this._Terms[phrase.ToLower()]];
			else
				return 0;
		}
		#endregion

		#region ctor
		/// <summary>
		/// 
		/// </summary>
		public TfIdfUtil()
		{
			this._Terms=new Dictionary<string, int>();
			this._DocLabels=new Dictionary<int, string>();
			this._DocTermFreqs=new Dictionary<int, Dictionary<int, int>>();
			this._TermWeights=new Dictionary<int, double>();
		}
		#endregion

		#region init 
		/// <summary>
		/// 
		/// </summary>
		/// <param name="label"></param>
		/// <param name="docContent"></param>
		public void AddDocument(string label, string docContent)
		{
			int lastDocID = 0;
			foreach(int docId in this._DocLabels.Keys)
			{
				if(docId>lastDocID)
					lastDocID = docId;
			}
			int newDocId = lastDocID + 1;
			this._DocLabels.Add(newDocId, label);
			this.IndexDocumentContent(newDocId, docContent);
		}

		/// <summary>
		/// 
		/// </summary>
		/// <param name="docId"></param>
		/// <param name="label"></param>
		/// <param name="docContent"></param>
		public void AddDocument(int docId, string label, string docContent)
		{
			if(this._DocLabels.ContainsKey(docId))
				throw new Exception("Doc is already added");
			this._DocLabels.Add(docId,docContent);
			this.IndexDocumentContent(docId, docContent);
		}

		private void IndexDocumentContent(int docId, string docContent)
		{
            docContent = docContent.ToLower();
            string[] phrases = docContent.Split(new char[] { ' ', '\n', '\r' });
            StopWordsUtil stopWordsUtil = new StopWordsUtil(StopWordsUtil.StopWordList.PubmedStopWordList);
            List<string> stopWords = stopWordsUtil.StopWords;
            Dictionary<int,int> termFreqs=new Dictionary<int, int>();
            PorterStemmer porterStemmer=new PorterStemmer();
            foreach (string phrase in phrases)
			{
                if (phrase.Trim().Length <= 3 || phrase.Trim().Length > 10)
                    continue;
                string stem = porterStemmer.StemTerm(phrase);

                if (stem.Length > 0 && !stopWords.Contains(stem))
				{
					if(this._Terms.ContainsKey(stem))
					{
						int termId = this._Terms[stem];
						if (termFreqs.ContainsKey(termId))
							termFreqs[termId] = termFreqs[termId] + 1;
						else 
							termFreqs.Add(termId,1);
					}
					else
					{
						int termId = this._Terms.Count + 1;
						this._Terms.Add(stem, termId);
						termFreqs.Add(termId,1);
						Dictionary<int, int> docFreqs = new Dictionary<int, int>();
						docFreqs.Add(docId, 1);
					}
				}
			}
			this._DocTermFreqs.Add(docId, termFreqs);
		}
		#endregion

		#region calculate 
		/// <summary>
		/// 
		/// </summary>
		public void ComputeTermWeights()
		{
			this._TermWeights=new Dictionary<int, double>();
			foreach(string phrase in this._Terms.Keys)
			{
				int termId = this._Terms[phrase];
				int docContainingTerm = 0;
				double tf = 0;
				foreach(int docId in this._DocTermFreqs.Keys)
				{
					if (this._DocTermFreqs[docId].ContainsKey(termId))
					{
						docContainingTerm++;
						int wordCountInDoc = 0;
						foreach (int termFreq in this._DocTermFreqs[docId].Values)
							wordCountInDoc += termFreq;
						tf += ((double) this._DocTermFreqs[docId][termId])/wordCountInDoc;
					}
				}
				double idf = Math.Log(((double)this._DocTermFreqs.Count)/docContainingTerm);
				this._TermWeights.Add(termId, tf*idf);
			}
		}
		#endregion 
	}
}
